"""
Question pre-processors.
"""


def clean_whitespace(question):
    """
    Remove any consecutive whitespace characters from the question text.
    """
    import re

    # Replace linebreaks and tabs with spaces
    question.text = question.text.replace('\n', ' ').replace('\r', ' ').replace('\t', ' ')

    # Remove any leading or trailing whitespace
    question.text = question.text.strip()

    # Remove consecutive spaces
    question.text = re.sub(r' +', ' ', question.text)

    return question


def unescape_html(question):
    """
    Convert escaped html characters into unescaped html characters.
    For example: "&lt;b&gt;" becomes "<b>".
    """
    import html

    question.text = html.unescape(question.text)

    return question


def convert_to_ascii(question):
    """
    Converts unicode characters to ASCII character equivalents.
    For example: "på fédéral" becomes "pa federal".
    """
    import unicodedata

    text = unicodedata.normalize('NFKD', statement.text)
    text = text.encode('ascii', 'ignore').decode('utf-8')

    question.text = str(text)
    return question
